import requests
from lxml import etree
from DAO.DAO_recruit_page import DAO_recruit_page
from DAO.DAO_university import DAO_university
import socket

class DATA_recruit_page:
    # 爬取所有学校的招生简章链接
    def university_link(self):
        urls = ['http://college.wendu.com/index.php?m=university&c=search&a=search&page={}'.format(str(i)) for i in range(1, 70)]
        headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"}
        for url in urls:
            print(url)
            response = requests.get(url, headers=headers)
            response_html = etree.HTML(response.content.decode())
            schools = response_html.xpath("//a[@class='school-list-itemLeft']")
            for school in schools:
                school_name = school.xpath(".//h3[@class='school-item-name']/text()")[0]
                school_link = "http://college.wendu.com" + school.xpath("./@href")[0]
                #print(school_link, school_name)
                self.recruit_page(school_link)
            #     break
            # break
    # 根据学校链接，爬取一个学校的招生简章信息，并存入数据库
    def recruit_page(self, url):
        #url = "https://college.wendu.com/detail-170-0-0"
        headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36"}
        socket.setdefaulttimeout(600)
        response = requests.get(url, headers=headers, timeout=600)
        response_html = etree.HTML(response.content.decode())
        #print(response.content)
        #test = response_html.xpath("//li[3]/div/table/tbody/text()")#/tr[2]/td[5]/a/@href
        # print(test)
        try:
            dao_recruit_page = DAO_recruit_page()
            dao_university = DAO_university()
            rec_uni_name = response_html.xpath("//div[@class='main_l fl ']/h2/div/@unname")[0]
            uni_code = response_html.xpath("//div[@class='main_l fl ']/h2//span/text()")[0].split("码")[1]
            if dao_recruit_page.not_uni_name(rec_uni_name):
                rec_year = response_html.xpath("//li[2]/div/h3/div/div[1]/text()")[0]
                contents = response_html.xpath("//li[2]/div/div//p/text()")
                #print(contents)
                rec_content = ''
                for conts in contents:
                    if len(conts.split()) > 0:
                        rec_cont = ''
                        for cont in conts.split():
                            rec_cont = rec_cont+cont
                        rec_content = rec_content+rec_cont+'<br>'
                print(rec_year, rec_content, rec_uni_name)
                dao_recruit_page.add(rec_year, rec_content, rec_uni_name)
            else:
                print(rec_uni_name, "已存在")
                if dao_university.uni_code_is0(rec_uni_name):
                    print(rec_uni_name, "的院校代码", uni_code, "刚刚更新")
                    dao_university.update_uni_code(uni_code, rec_uni_name)
                else:
                    print("院校", rec_uni_name, "的代码不需要更新")

        except Exception as e:
            print(e)
            print("此处出错")


if __name__ == '__main__':
    print("1、爬取所有院校招生简章数据")
    print("2、爬取一个院校招生简章数据")
    print("3、清空数据库(待定)")
    print("4、输出所有数据")
    res = int(input())
    if res  == 1:
        # 爬取所有院校招生简章数据
        test = DATA_recruit_page()
        test.university_link()
    elif res == 2:
        test = DATA_recruit_page()
        test.recruit_page("http://college.wendu.com/detail-317-0-0")
    elif res == 3:
        pass
        dao = DAO_recruit_page()
        dao.truncate()
    elif res == 4:
        dao = DAO_recruit_page()
        dao.sel_all()
